''' 
@文件：3.使用进程池爬取芒果电影信息.py
@作者：Miss丶念
@时间：2025/3/22：22:05
'''

import requests
import redis
import pymongo
import hashlib
from concurrent.futures import ProcessPoolExecutor, as_completed


class MangoSpider:
    redis_cli = redis.Redis()
    mongo_client = pymongo.MongoClient()
    mongo_db = mongo_client['pyspider']['ProcessPoolExecutor_mango_movie']

    def __init__(self):
        self.url = 'https://pianku.api.mgtv.com/rider/list/pcweb/v3'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
            'Referer': 'https://www.mgtv.com/'
        }

    @staticmethod
    # 上传翻页参数到队列
    def put_params():
        param_dict = []
        for page in range(1, 6):
            params = {
                "allowedRC": "1",
                "platform": "pcweb",
                "channelId": "2",
                "pn": page,
                "pc": "80",
                "hudong": "1",
                "_support": "10000000",
                "kind": "19",
                "area": "10",
                "year": "all",
                "chargeInfo": "a1",
                "sort": "c2",
                "feature": "all"
            }
            param_dict.append(params)
        return param_dict

    # 请求数据
    def get_data(self, params):
        response = requests.get(self.url, headers=self.headers, params=params).json()
        return response

    def parse_data(self, response):
        items = []
        movie_list = response['data']['hitDocs']
        for movie in movie_list:
            item = dict()
            item['title'] = movie['title']
            item['subtitle'] = movie['subtitle']
            item['story'] = movie['story']
            items.append(item)
        return items

    # 数据去重
    @staticmethod
    def get_md5(content):
        md5 = hashlib.md5(str(content).encode("utf-8")).hexdigest()
        return md5

    # 数据保存
    def save_data(self, items):
        for item in items:
            parse_data = self.get_md5(item)
            result = self.redis_cli.sadd('mango_movie:filter', parse_data)
            if result:
                self.mongo_db.insert_one(item)
                print("数据保存成功：", item)
            else:
                print("数据已存在：", item)

    # 关闭数据库链接
    def close_spider(self):
        print('爬虫即将退出, 准备关闭数据库链接...')
        self.mongo_client.close()
        self.redis_cli.close()

    # 启动函数
    def main(self):
        # 获取翻页参数列表
        params_list = self.put_params()
        with ProcessPoolExecutor(max_workers=5) as executor:
            # 提交请求数据的任务
            future_to_params = {executor.submit(self.get_data, params): params for params in params_list}
            # 处理请求数据的结果
            for future in as_completed(future_to_params):
                response = future.result()
                items = self.parse_data(response)
                self.save_data(items)
        # 关闭数据库链接
        self.close_spider()
        print('爬虫任务完成...')


if __name__ == '__main__':
    mango_spider = MangoSpider()
    mango_spider.main()
